# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem
from scrapy.http import Request
import re,os
from scrapy.utils.misc import md5sum

class MyspiderPipeline(object):
    def process_item(self, item, spider):
        return item

class ImgPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        for img_url in item['url']:
            yield Request(img_url,meta={'item':item['name']})

    def file_path(self, request, response=None, info=None):
        name = request.meta['item']
        image_guid = request.url.split('/')[-1]
        filename = u'full/{0}/{1}'.format(name, image_guid)
        return filename

    def image_downloaded(self, response, request, info):
        checksum = None
        for path, image, buf in self.get_images(response, request, info):
            if checksum is None:
                buf.seek(0)
                checksum = md5sum(buf)

            absolute_path = self.get_filesystem_path(path)
            self.mkdir(os.path.dirname(absolute_path), info)
            with open(absolute_path, 'wb') as f:
                f.write(response.body)
            # width, height = image.size
            # self.store.persist_file(
            #     path, buf, info,
            #     meta={'width': width, 'height': height},
            #     headers={'Content-Type': 'image/jpeg'})
        return checksum

    def get_filesystem_path(self, path):
        path_comps = path.split('/')
        return os.path.join('D:\meizi2', *path_comps)

    def mkdir(self, dirname, domain=None):
        from collections import defaultdict
        self.created_directories = defaultdict(set)
        seen = self.created_directories[domain] if domain else set()
        if dirname not in seen:
            if not os.path.exists(dirname):
                os.makedirs(dirname)
            seen.add(dirname)
    # def item_completed(self, results, item, info):
    #     image_path = [x['path'] for ok, x in results if ok]
    #     if not image_path:
    #         raise DropItem('Item contains no images')
    #     item['image_paths'] = image_path
    #     return item

